import pymysql
from configs.cfg import *

conn = pymysql.connect(host=mysql_host, user=mysql_name, password=mysql_pwd,
                       database='spiderdb', port=mysql_port, charset='utf8')

cur = conn.cursor()
start_count = 0
end_count = 10000
for i in range(24468003):
    if end_count <= 24468003:
        sql = f'SELECT distinct website FROM websiteaddresses limit {start_count},{end_count}'
        cur.execute(sql)
        datas = cur.fetchall()
        start_count += 10000
        end_count += 10000
    else:
        break
